import scrapy
import logging
import base64

term = "20161"
collegeAmount = 37
urlPrefix = "http://192.168.240.168/xuanke/coursehtm/d"
urlSuffix = term + ".htm"

class CourseSys( scrapy.Spider ):
        http_user = ''
        http_pass = ''
        http_proxy = 'http://proxy.szu.edu.cn:8080'
	name = "courses"
	def start_requests( self ):
            en = "Basic "+base64.encodestring(self.http_user+':'+self.http_pass)
	    for x in range(1, collegeAmount):
		url = urlPrefix + str(x) + urlSuffix
                yield scrapy.Request( url=url, callback=self.parseCourse, meta={"proxy":self.http_proxy},headers={ "Proxy-Authorization":en })


	def parseCourse( self, response ):
		table = response.css('body > table');
		college = table.css('tr:nth-child(1) > td > font::text');
		courses = table.css('tr:nth-of-type(n+3)')
		parsedCourses = []
		for course in courses:
			parsedCourse = {}
			courseTds = course.css('td')
			parsedCourse["college"] = college[0:-4]
			parsedCourse['id'] = courseTds[2].extract()
			parsedCourse["name"] = courseTds[3].extract()
			parsedCourse["mainclass"] = courseTds[4].extract()
			parsedCourse["teacher"] = courseTds[5].extract()
			parsedCourse["credits"] = courseTds[6].extract()
			parsedCourse["classroom"] = courseTds[9].extract()
			parsedCourse["sjd"] = courseTds[10].extract()
			parsedCourse["creditType"] = courseTds[11].extract()
			parsedCourse["bz"] = courseTds[12].extract()
			yield parsedCourse
			
